/*
 *  Ranker.h
 *  OfflineTranslator
 *
 *  Created by Petr Homola on 7.10.10.
 *  Copyright 2010 Codesign LLC. All rights reserved.
 *
 */

#ifndef NLP_RANKER_H_
#define NLP_RANKER_H_

inline unsigned int rankerHash(const char* key) {
	unsigned int _hashCode = 17;
	while (*key) { _hashCode = _hashCode * 37 + *key; key++; }
	return _hashCode;
}

inline unsigned int rankerBiHash(const char* history, const char* key) {
	return ((rankerHash(key) << 16) | (rankerHash(history) % 65536));
}

#include "global.h"
#include <iostream>
#include <string>
#include <map>
#include <vector>
#include <assert.h>
#include <math.h>
#ifdef TR1
#include <tr1/unordered_map>
#else
#include <unordered_map>
#endif

namespace nlp {
	
	using namespace std;
#ifdef TR1
	using namespace std::tr1;
#endif
	
	class Ranker {
	private:
		unordered_map<unsigned short,unsigned short> unigrams;
		unordered_map<unsigned int,unsigned short> bigrams;
	public:
		static const unsigned int dataLength = 46000000;
		Ranker(const char* unigramsFileName, const char* bigramsFileName) {
			unigrams.rehash(65600);
			bigrams.rehash(376000);
			cout << "unigr. " << unigrams.bucket_count() << " " << unigrams.load_factor() << endl;
			cout << "bigr.  " << bigrams.bucket_count() << " " << bigrams.load_factor() << endl;
			
			FILE* file = fopen(unigramsFileName, "rb");
			assert(file != NULL);
			int n = 0;
			while (!feof(file)) {
				unsigned short hash, count;
				fread(&hash, sizeof(unsigned short), 1, file);
				if (feof(file)) {
					cout << "read " << n << " hashed n-grams" << endl;
					break;
				}
				n++;
				assert(!feof(file));
				fread(&count, sizeof(unsigned short), 1, file);
				unigrams[hash] = count;
			}
			fclose(file);
			
			file = fopen(bigramsFileName, "rb");
			assert(file != NULL);
			n = 0;
			while (!feof(file)) {
				unsigned int hash;
				unsigned short count;
				fread(&hash, sizeof(unsigned int), 1, file);
				if (feof(file)) {
					cout << "read " << n << " hashed n-grams" << endl;
					break;
				}
				n++;
				assert(!feof(file));
				fread(&count, sizeof(unsigned short), 1, file);
				//hash &= 65536 * 4 - 1;
				bigrams[hash] = count;
			}
			fclose(file);
			
			cout << "##### " << unigrams.size() << " " << bigrams.size() << endl;
			cout << "unigr. " << unigrams.bucket_count() << " " << unigrams.load_factor() << endl;
			cout << "bigr.  " << bigrams.bucket_count() << " " << bigrams.load_factor() << endl;
		}
		double rank(const char** tokens, unsigned int length) {
			double score = 0;
			unsigned short prevCount;
			//unsigned int prevHash;
			for (int i = 0; i < length; i++) {
				unsigned int hash = rankerHash(tokens[i]) % 65536;
				unsigned short count = unigrams[hash];
				if (i > 0) {
					unsigned short bicount = bigrams[rankerBiHash(tokens[i - 1], tokens[i]) % (65536 * 8)];
					assert(bicount <= prevCount);
					if (bicount > 0) {
						double prob = bicount / (double) prevCount;
						score -= log(prob);
					} else {
						//cout << "unigram fallback" << endl;
						double prob = count / (double) dataLength;
						score -= log(prob);
					}
				} else {
					double prob = count / (double) dataLength;
					score -= log(prob);
				}
				prevCount = count;
				//prevHash = hash;
			}
			return score;
		}
	};

}

#endif
